import os
from utils.FileReader import FileReader
from dataprocess.task.DocsIngestTask import process_document
from utils.FileReader import FileReader



root_dir = r'G:\code\cursor\web-monitor-webdriver\all_luobaizhu_related_pages'
processed_count = 0
for root, _, files in os.walk(root_dir):
    for file in files:
      if file.endswith(".txt"):
        # 构建文件的绝对路径
        absolute_path = os.path.join(root, file)
        # 计算相对路径
        relative_path = os.path.relpath(absolute_path, root_dir)
        cotent = FileReader().read(absolute_path)
        print(relative_path)
        messages = {'file_path':absolute_path, 'file_name': relative_path}
        process_document(messages)
        processed_count = processed_count + 1
        print(f"processed: {processed_count}")
